{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using a model of type albert to instantiate a model of type bert. This is not supported for all configurations of models and can yield errors.\n",
      "Some weights of the model checkpoint at voidful/albert_chinese_small were not used when initializing BertModel: ['predictions.bias', 'albert.encoder.albert_layer_groups.0.albert_layers.0.attention.value.bias', 'predictions.decoder.weight', 'albert.encoder.albert_layer_groups.0.albert_layers.0.attention.dense.weight', 'albert.encoder.albert_layer_groups.0.albert_layers.0.full_layer_layer_norm.bias', 'predictions.LayerNorm.weight', 'albert.encoder.albert_layer_groups.0.albert_layers.0.attention.LayerNorm.weight', 'albert.encoder.albert_layer_groups.0.albert_layers.0.attention.key.bias', 'albert.encoder.albert_layer_groups.0.albert_layers.0.ffn_output.bias', 'albert.embeddings.word_embeddings.weight', 'albert.embeddings.token_type_embeddings.weight', 'albert.embeddings.LayerNorm.weight', 'albert.encoder.embedding_hidden_mapping_in.weight', 'albert.pooler.weight', 'albert.encoder.embedding_hidden_mapping_in.bias', 'albert.encoder.albert_layer_groups.0.albert_layers.0.attention.dense.bias', 'predictions.dense.bias', 'predictions.LayerNorm.bias', 'albert.encoder.albert_layer_groups.0.albert_layers.0.attention.key.weight', 'albert.embeddings.LayerNorm.bias', 'albert.encoder.albert_layer_groups.0.albert_layers.0.ffn.weight', 'albert.encoder.albert_layer_groups.0.albert_layers.0.attention.value.weight', 'albert.pooler.bias', 'albert.encoder.albert_layer_groups.0.albert_layers.0.attention.query.weight', 'predictions.dense.weight', 'predictions.decoder.bias', 'albert.embeddings.position_embeddings.weight', 'albert.encoder.albert_layer_groups.0.albert_layers.0.full_layer_layer_norm.weight', 'albert.encoder.albert_layer_groups.0.albert_layers.0.attention.LayerNorm.bias', 'albert.encoder.albert_layer_groups.0.albert_layers.0.ffn_output.weight', 'albert.encoder.albert_layer_groups.0.albert_layers.0.attention.query.bias', 'albert.encoder.albert_layer_groups.0.albert_layers.0.ffn.bias']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertModel were not initialized from the model checkpoint at voidful/albert_chinese_small and are newly initialized: ['encoder.layer.5.attention.self.key.bias', 'encoder.layer.5.attention.output.LayerNorm.weight', 'encoder.layer.4.attention.self.value.bias', 'encoder.layer.0.output.LayerNorm.bias', 'encoder.layer.2.attention.output.dense.weight', 'embeddings.position_embeddings.weight', 'encoder.layer.1.attention.self.query.bias', 'encoder.layer.4.intermediate.dense.weight', 'encoder.layer.1.attention.output.LayerNorm.bias', 'encoder.layer.3.attention.self.key.bias', 'encoder.layer.4.attention.output.LayerNorm.bias', 'encoder.layer.2.attention.self.key.weight', 'encoder.layer.3.output.dense.weight', 'encoder.layer.0.attention.self.value.bias', 'encoder.layer.0.intermediate.dense.bias', 'encoder.layer.4.attention.self.key.weight', 'encoder.layer.2.attention.self.value.weight', 'encoder.layer.0.attention.self.query.bias', 'encoder.layer.5.output.dense.bias', 'pooler.dense.bias', 'encoder.layer.2.attention.output.LayerNorm.bias', 'encoder.layer.2.output.LayerNorm.weight', 'encoder.layer.3.attention.output.LayerNorm.bias', 'encoder.layer.5.intermediate.dense.weight', 'encoder.layer.4.intermediate.dense.bias', 'embeddings.LayerNorm.weight', 'encoder.layer.0.intermediate.dense.weight', 'encoder.layer.3.intermediate.dense.bias', 'encoder.layer.3.attention.self.value.bias', 'encoder.layer.2.intermediate.dense.bias', 'encoder.layer.2.output.LayerNorm.bias', 'encoder.layer.3.attention.self.query.bias', 'encoder.layer.2.attention.self.value.bias', 'encoder.layer.4.attention.self.key.bias', 'encoder.layer.5.attention.output.dense.bias', 'encoder.layer.3.attention.self.value.weight', 'encoder.layer.0.attention.self.key.weight', 'encoder.layer.1.output.LayerNorm.weight', 'encoder.layer.1.attention.self.value.bias', 'encoder.layer.1.attention.output.dense.weight', 'encoder.layer.5.attention.self.value.weight', 'encoder.layer.1.output.dense.weight', 'encoder.layer.4.output.LayerNorm.bias', 'encoder.layer.2.intermediate.dense.weight', 'embeddings.token_type_embeddings.weight', 'encoder.layer.1.attention.self.key.bias', 'encoder.layer.5.output.dense.weight', 'encoder.layer.1.attention.output.LayerNorm.weight', 'encoder.layer.0.attention.self.query.weight', 'encoder.layer.4.attention.self.value.weight', 'encoder.layer.1.intermediate.dense.weight', 'encoder.layer.4.attention.output.dense.bias', 'encoder.layer.0.attention.output.LayerNorm.bias', 'encoder.layer.1.intermediate.dense.bias', 'encoder.layer.3.output.dense.bias', 'encoder.layer.5.attention.self.query.weight', 'encoder.layer.0.attention.output.dense.bias', 'encoder.layer.5.attention.self.value.bias', 'encoder.layer.2.output.dense.weight', 'encoder.layer.4.attention.self.query.weight', 'encoder.layer.5.attention.output.LayerNorm.bias', 'encoder.layer.3.attention.self.key.weight', 'encoder.layer.0.output.LayerNorm.weight', 'encoder.layer.2.attention.self.query.weight', 'encoder.layer.0.output.dense.bias', 'encoder.layer.4.attention.output.dense.weight', 'encoder.layer.1.attention.self.key.weight', 'encoder.layer.2.attention.self.query.bias', 'encoder.layer.5.intermediate.dense.bias', 'encoder.layer.3.attention.output.dense.bias', 'encoder.layer.4.attention.self.query.bias', 'encoder.layer.4.output.LayerNorm.weight', 'encoder.layer.5.attention.output.dense.weight', 'encoder.layer.2.attention.self.key.bias', 'encoder.layer.3.attention.output.LayerNorm.weight', 'encoder.layer.1.output.dense.bias', 'encoder.layer.4.output.dense.bias', 'encoder.layer.0.attention.self.value.weight', 'encoder.layer.3.output.LayerNorm.bias', 'encoder.layer.2.attention.output.dense.bias', 'encoder.layer.2.output.dense.bias', 'encoder.layer.5.output.LayerNorm.weight', 'encoder.layer.0.attention.self.key.bias', 'encoder.layer.5.attention.self.query.bias', 'pooler.dense.weight', 'encoder.layer.3.intermediate.dense.weight', 'encoder.layer.1.output.LayerNorm.bias', 'encoder.layer.1.attention.output.dense.bias', 'encoder.layer.0.output.dense.weight', 'encoder.layer.1.attention.self.value.weight', 'embeddings.LayerNorm.bias', 'encoder.layer.3.attention.self.query.weight', 'encoder.layer.3.attention.output.dense.weight', 'encoder.layer.5.attention.self.key.weight', 'encoder.layer.4.attention.output.LayerNorm.weight', 'encoder.layer.4.output.dense.weight', 'encoder.layer.3.output.LayerNorm.weight', 'encoder.layer.1.attention.self.query.weight', 'encoder.layer.2.attention.output.LayerNorm.weight', 'embeddings.word_embeddings.weight', 'encoder.layer.5.output.LayerNorm.bias', 'encoder.layer.0.attention.output.LayerNorm.weight', 'encoder.layer.0.attention.output.dense.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "You are using a model of type albert to instantiate a model of type bert. This is not supported for all configurations of models and can yield errors.\n"
     ]
    }
   ],
   "source": [
    "# encoding: utf-8\n",
    "# 使用Albert进行文本分类\n",
    "# url https://blog.csdn.net/u013230189/article/details/108836511\n",
    "\n",
    "import pandas as pd\n",
    "import torch\n",
    "from transformers import BertTokenizer, BertModel, BertConfig\n",
    "import numpy as np\n",
    "from torch.utils import data\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 加载预训练模型\n",
    "pretrained = 'voidful/albert_chinese_small'\n",
    "tokenizer = BertTokenizer.from_pretrained(pretrained)\n",
    "model = BertModel.from_pretrained(pretrained)\n",
    "config = BertConfig.from_pretrained(pretrained)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AlbertClassfier(torch.nn.Module):\n",
    "    def __init__(self, bert_model, bert_config, num_class):\n",
    "        super(AlbertClassfier, self).__init__()\n",
    "        self.bert_model = bert_model\n",
    "        self.dropout = torch.nn.Dropout(0.4)\n",
    "        self.fc1 = torch.nn.Linear(bert_config.hidden_size, bert_config.hidden_size)\n",
    "        self.fc2 = torch.nn.Linear(bert_config.hidden_size, num_class)\n",
    "\n",
    "    def forward(self, token_ids):\n",
    "        bert_out = self.bert_model(token_ids)[1]  # 句向量 [batch_size,hidden_size]\n",
    "        bert_out = self.dropout(bert_out)\n",
    "        bert_out = self.fc1(bert_out)\n",
    "        bert_out = self.dropout(bert_out)\n",
    "        bert_out = self.fc2(bert_out)  # [batch_size,num_class]\n",
    "        return bert_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "albertBertClassifier = AlbertClassfier(model, config, 2)\n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else 'cpu'\n",
    "albertBertClassifier = albertBertClassifier.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_test_data(pos_file_path, neg_file_path, max_length=100, test_size=0.2):\n",
    "    data = []\n",
    "    label = []\n",
    "    with open(pos_file_path, 'r', encoding='utf-8') as f:\n",
    "        pos = f.readlines()\n",
    "    for p in pos:\n",
    "        ids = tokenizer.encode(p.strip(), max_length=max_length, padding=\"max_length\", truncation='longest_first')\n",
    "        data.append(ids)\n",
    "        label.append(1)\n",
    "\n",
    "    # up pos, down neg\n",
    "    with open(neg_file_path, 'r', encoding='utf-8') as f:\n",
    "        neg = f.readlines()\n",
    "    for n in neg:\n",
    "        ids = tokenizer.encode(n.strip(), max_length=max_length, padding=\"max_length\", truncation='longest_first')\n",
    "        data.append(ids)\n",
    "        label.append(0)\n",
    "\n",
    "    X_train, X_test, y_train, y_test = train_test_split(data, label, test_size=test_size, shuffle=True)\n",
    "    return (X_train, y_train), (X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos_file_path = \"./pos.txt\"\n",
    "neg_file_path = \"./neg.txt\"\n",
    "(X_train, y_train), (X_test, y_test) = get_train_test_data(pos_file_path, neg_file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataloader\n",
    "class DataGen(data.Dataset):\n",
    "    def __init__(self, data, label):\n",
    "        self.data = data\n",
    "        self.label = label\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        data = torch.Tensor(self.data[index])\n",
    "        label = torch.Tensor(self.label[index])\n",
    "        return data, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = DataGen(X_train, y_train)\n",
    "test_dataset = DataGen(X_test, y_test)\n",
    "train_dataloader = data.DataLoader(train_dataset, batch_size=256)\n",
    "test_dataloader = data.DataLoader(test_dataset, batch_size=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(albertBertClassifier.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "stack expects each tensor to be equal size, but got [0] at entry 0 and [1] at entry 2",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_16096\\932878107.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m     \u001b[0maccu\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0malbertBertClassifier\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[1;32mfor\u001b[0m \u001b[0mstep\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtoken_ids\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_dataloader\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m         \u001b[0mtoken_ids\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtoken_ids\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m         \u001b[0mlabel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\torch\\lib\\site-packages\\torch\\utils\\data\\dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    519\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    520\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 521\u001b[1;33m             \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    522\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    523\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[1;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\torch\\lib\\site-packages\\torch\\utils\\data\\dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    559\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    560\u001b[0m         \u001b[0mindex\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# may raise StopIteration\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 561\u001b[1;33m         \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# may raise StopIteration\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    562\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    563\u001b[0m             \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\torch\\lib\\site-packages\\torch\\utils\\data\\_utils\\fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[1;34m(self, possibly_batched_index)\u001b[0m\n\u001b[0;32m     50\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     51\u001b[0m             \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 52\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcollate_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\.conda\\envs\\torch\\lib\\site-packages\\torch\\utils\\data\\_utils\\collate.py\u001b[0m in \u001b[0;36mdefault_collate\u001b[1;34m(batch)\u001b[0m\n\u001b[0;32m     82\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'each element in list of batch should be of equal size'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     83\u001b[0m         \u001b[0mtransposed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 84\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mdefault_collate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0msamples\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtransposed\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     85\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     86\u001b[0m     \u001b[1;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdefault_collate_err_msg_format\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0melem_type\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\torch\\lib\\site-packages\\torch\\utils\\data\\_utils\\collate.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m     82\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'each element in list of batch should be of equal size'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     83\u001b[0m         \u001b[0mtransposed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 84\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mdefault_collate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0msamples\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtransposed\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     85\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     86\u001b[0m     \u001b[1;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdefault_collate_err_msg_format\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0melem_type\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\torch\\lib\\site-packages\\torch\\utils\\data\\_utils\\collate.py\u001b[0m in \u001b[0;36mdefault_collate\u001b[1;34m(batch)\u001b[0m\n\u001b[0;32m     54\u001b[0m             \u001b[0mstorage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0melem\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstorage\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_new_shared\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnumel\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     55\u001b[0m             \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0melem\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnew\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstorage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 56\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     57\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0melem_type\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__module__\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'numpy'\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0melem_type\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34m'str_'\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m             \u001b[1;32mand\u001b[0m \u001b[0melem_type\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34m'string_'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: stack expects each tensor to be equal size, but got [0] at entry 0 and [1] at entry 2"
     ]
    }
   ],
   "source": [
    "\n",
    "for epoch in range(50):\n",
    "    loss_sum = 0.0\n",
    "    accu = 0\n",
    "    albertBertClassifier.train()\n",
    "    for step, (token_ids, label) in enumerate(train_dataloader):\n",
    "        token_ids = token_ids.to(device)\n",
    "        label = label.to(device)\n",
    "        out = albertBertClassifier(token_ids)\n",
    "        loss = criterion(out, label)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()  # 反向传播\n",
    "        optimizer.step()  # 梯度更新\n",
    "        loss_sum += loss.cpu().data.numpy()\n",
    "        accu += (out.argmax(1) == label).sum().cpu().data.numpy()\n",
    "\n",
    "    test_loss_sum = 0.0\n",
    "    test_accu = 0\n",
    "    albertBertClassifier.eval()\n",
    "    for step, (token_ids, label) in enumerate(test_dataloader):\n",
    "        token_ids = token_ids.to(device)\n",
    "        label = label.to(device)\n",
    "        with torch.no_grad():\n",
    "            out = albertBertClassifier(token_ids)\n",
    "            loss = criterion(out, label)\n",
    "            test_loss_sum += loss.cpu().data.numpy()\n",
    "            test_accu += (out.argmax(1) == label).sum().cpu().data.numpy()\n",
    "    print(\"epoch % d,train loss:%f,train acc:%f,test loss:%f,test acc:%f\" % (\n",
    "        epoch, loss_sum / len(train_dataset), accu / len(train_dataset), test_loss_sum / len(test_dataset),\n",
    "        test_accu / len(test_dataset)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
